/*
* @Author: steven.wang
* @Date:   2019-03-20 16:49:03
* @Last Modified by:   steven.wang
* @Last Modified time: 2019-03-23 10:44:21
*/
const fs = require("fs");
const cheerio = require('cheerio');
const asyncQuene = require("async").queue;
const request = require('superagent');
require('superagent-charset')(request);
const sleep = require('sleep');
const tools = require('./tools');
const config = require('./config');

/**
 * 获取所有job的链接
 * @author steven.wang
 * @DateTime 2019-03-23T00:57:37+0800
 */
const getJobUrls = () => {
    return new Promise((resolve, reject) => {
        console.log('Start get jobs .....');
        let jobs = [], errors = [], count = 0;
        let q = asyncQuene(async (pageUrl, cb) => {
            // if(count%100 == 1) {
            //     sleep.sleep(5);
            // }
            // count++;
            try {
                let $ = await tools.getHtmlAsync(pageUrl.url);
                console.log(`download ${pageUrl.url} success`);
                $('.sojob-list>li').each(function(idx, elem) {
                    let url = $(this).find('.job-info>h3>a').attr('href');
                    jobs.push({
                        type: pageUrl.name,
                        name: $(this).find('.job-info>h3').text().trim(),
                        salary: $(this).find('.job-info>.condition>.text-warning').text(),
                        company: $(this).find('.company-info>.company-name').text().trim(),
                        url: url.substring(0,1) == '/' ? config.liepin.host + url : url,
                    });
                });
            } catch (err) {
                errors.push(pageUrl);
                console.log(`Error : get job list - download ${pageUrl.url} err : ${err}`);
            } finally {
                cb();
            }
        }, config.concurrent);
        /**
         * 监听：当所有任务都执行完以后，将调用该函数
         */
        q.drain = () => {
            if (errors.length > 0) {
                console.log('Get job url err: ' + JSON.stringify(errors));
                tools.writeJsonToFile('files/liepin-job-url-error.json', errors);
            }
            console.log('Get job url complete');
            resolve(jobs);
        }

        let pageUrls = [];
        for (let item of config.liepin.list) {
            for (let subItem of item.children) {
                for (let i = subItem.startPage; i <= subItem.endPage; i++) {
                    pageUrls.push({
                        name: subItem.name,
                        url: subItem.url + `&curPage=${i}&d_curPage=${i-1}&d_pageSize=40`
                    });
                }
            }
        }
        q.push(pageUrls);
    });
}
/**
 * 获取所有job的详情
 * @author steven.wang
 * @DateTime 2019-03-23T00:57:59+0800
 */
const getJobsDetail = data => {
    return new Promise((resolve, reject) => {
        console.log('Start get job detail .....');
        let jobs = [], errors = [], count = 0;
        let q = asyncQuene(async (pageUrl, cb) => {
            // if(count%100 == 1) {
            //     sleep.sleep(5);
            // }
            // count++;
            try {
                let $ = await tools.getHtmlAsync(pageUrl.url);
                console.log(`download ${pageUrl.url} success`);
                jobs.push({
                    source: '猎聘网',
                    name: $('.about-position>.title-info>h1').html(),
                    state: '招聘中',
                    type: pageUrl.type,
                    label: (function () {
                        let result = [];
                        $('.comp-tag-list>li').find('span').each(function() {
                            result.push($(this).html())
                        });
                        return result.join(',');
                    })(),
                    work_time: '全职',
                    publish_time: $('.basic-infor>time').text().trim(),
                    salary: pageUrl.salary,
                    city: $('.basic-infor>span').text().trim().split('-')[0],
                    experience: $('.job-qualifications>span').eq(1).text(),
                    education: $('.job-qualifications>span').eq(0).text(),
                    advantage: (function () {
                            let result = [];
                            $('.comp-tag-list').find('span').each(function() {
                                result.push($(this).html())
                            });
                            return result.join(',');
                        })(),
                    desc: $('.job-description .content').html(),
                    address: $('.new-compintro>li').eq(2).html() ? $('.new-compintro>li').eq(2).html().replace(/公司地址：/g, '') : $('.basic-infor>span').text().trim(),
                    hr_name: $('.publisher-name>span').text() || $('.publisher-name').next().text(),
                    hr_position: $('.publisher-name>.muted').text() ? $('.publisher-name>.muted').text().replace(/\/\ /g, '') : '',
                    company_name: $('.company-logo>p').text().trim() || $('.about-position>.title-info>h3').text().trim(),
                    company_id: '',
                    company_detail_url: $('.company-logo>a').attr('href') || ''
                });
            } catch (err) {
                errors.push(pageUrl);
                console.log(`Error : get job list - download ${pageUrl.url} err : ${err}`);
            } finally {
                cb();
            }
        }, config.concurrent);
        /**
         * 监听：当所有任务都执行完以后，将调用该函数
         */
        q.drain = () => {
            if (errors.length > 0) {
                console.log('Get job detail err: ' + JSON.stringify(errors));
                tools.writeJsonToFile('files/liepin-job-detail-error.json', errors);
            }
            console.log('Get job detail complete');
            resolve(jobs);
        }

        q.push(data);
    });
}

async function app() {
    let jobList, jobs = [];
    // if (fs.existsSync(config.liepin.jobDetailPath)) {
    //     jobList = JSON.parse(fs.readFileSync(config.liepin.jobDetailPath));
    // }
    jobList = await getJobUrls();
    tools.writeJsonToFile(config.liepin.jobUrlPath, jobList);
    console.log(jobList.length);

    jobList = await getJobsDetail(jobList);
    tools.writeJsonToFile(config.liepin.jobDetailPath, jobList);

    for (let item of jobList) {
        jobs.push([item.source || '', item.name || '', item.state || '', item.type || '',
            item.label || '', item.work_time || '', item.publish_time || '', item.salary || '',
            item.city || '', item.experience || '', item.education || '', item.advantage || '',
            item.description || '', item.address || '', item.hr_name || '', item.hr_position || '',
            item.company_name || '', item.company_id || '', item.company_detail_url]);
    }
    // 存入数据库
    let insertResult = await tools.insertJob(jobs);
    console.log(insertResult)
}

app();
